package com.example.demo.util;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.ctrip.framework.apollo.ConfigChangeListener;
import com.ctrip.framework.apollo.model.ConfigChange;
import com.ctrip.framework.apollo.model.ConfigChangeEvent;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.TimeUnit;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;


public class XiaohongshuCrawler {
    private static final Logger log = LoggerFactory.getLogger(XiaohongshuCrawler.class);
    //private static final String B1 = "I38rHdgsjopgIvesdVwgIC+oIELmBZ5e3VwXLgFTIxS3bqwErFeexd0ekncAzMFYnqthIhJeDdRxqqwdIvAsWBchwPttgm46KUzxIh0s0LKsjqwny7DZIvosxbPjycuAIhKeDa7sfUTTJqtR+qwApdpNIChmNW8KzVwSICgekutHwqwDIEFqIE/sTIZiIxrMIvgedqwR+7pPzZNsdY3efqtAIkNe3VwgqqtpputxKzgs1WpeICVWIhQ0/PtQKVwAI3Z2omdeiqwfICAeTVtrIE++IEmncfdejgzYroosfqwOZWoedLDDBogsjZpBIxKsdVtaIkvefqwRbMLIGf4BI37eTutS/SKe17MPIhbsIEM+wn4iI3Yq8lEgIkLxoqwkICqV2d3ejIgs1uwRIvge0fAejAuPIi/eTpmDyuwuIiKeTf0sxm/e1Vt4LsJeWVwCIvkgaA7efPw5gfOsDuwgIxQrIvkkoVwGzVw+ag4QPW/edngeTVwoIEosjBAsxLrvIE0s0L3s6fGdIhNs3uwvIEmWtuwpOqwCI3JeTVtFIk3siqwVIEosfVtANPw8mPwMIvP5IxI/8oHW2Zge3qtMIhIcIideDPwzrzzK/U6sYa+xIirm2qtwmPwRIv0exqtYtF/eTm7sTLSmIhAs3g/siqt/nuw+IC0ejqwRIi/eV0WmIv7eSnOeSPtAIxvs6Pwu2fWBIEcnICgsVM6s6eesSI3eDVtQIvcRI3RXmPtcIC4MIE/sxbHvHj5sSutmIEmOZqtEJZJsDp6eYuwmmuwgIvE7IirNpVw48qtzIC3efb0skPtgZqwHIicNcqwdICbaLBY+IE5efj0e0/7sVutq4qwPIEYsIv0e3qw9IigsjIhAIhZJsl+NIvuzIx6exutCLPw+OqwFIxlTIEOe1gosTuwdmZR3IxmywVwepPwBIv+hICF8IEuYIkvsTZ5ekcG1KqtyIxiDrVtPLqtvIkds1ASXIEzkrU6sjPwxguwszLzNgjbRIhFDeutqIEveiLp+IEWQput6IEYDNPtYrqttIEWfIh4vICbbeuw9IkzoIioekVwhsZEEIEZoICosY7NeVVt0IxI9IiEjsrTXIxileS8UICNeWDJeDPwbIEJsiutKIvVgICosfVwoIvYn/PtYI3veVMGyzPtl+VteICAejPtOIE7edlDvIioeWzJe3VwL2B/ejutt/mgs3VwqI3JsTm5eTVtsIEPzIiYsoPw+OmWGIvgs0Pt/a7osiWT0ButsIx4NIiOsfWIJICNsjPwzIxmqIiKsWWr1Ii4aIEMyKIS8Iv/eYlrxIhl4IxIYICOsdalG8uw/ruwzzp7sdFesWutWIvKedFOeTuw/bPwgIvZ2oVwTBVt/GuwT+qtSyVwWqU6eipvsi/5e6rvs3VtvIEMBIkuxICcFI3oe37oe6srxHVtzIEMpoPtyw/ds1uwqICosfl6sfniJIx5edutPGIc6IvAsYqw0I3oe3VtxKqwwIEosYUSr2Pw4oB3efYmiIh6sDutnnuwD/9Ks6Pw1ButZaVwQIvV8IkkRBLKsdzdsTVtdIxee1Pw2nPwbt0ge1STTcF+uQSGEZqwbmqttIx4fIxr4ICbT4cJsieiqIv4qIvW2ruwQI3zYJqwPIkeeVIIjoVtdIhbnguwHIvAexLIaIkzZIC0sfuwtpqtzpzJeSs3sfPtb4qwmICrT2uwcIE3eWU/eDede1VwKICPpIEgeT/pZIx6eYVwmIv+uIvveWqwzOrW2sqtLcPwoICpdLVwgIv3s0Z0e0uwZIhoekeQbIC3sWmosjqtpIhZaIvza/e/sdm3sjqwOIiosjU0eTPwfmPt2nVwPIiWNIxveSVt6IxknICRhQuwlwUMBwzvsxUmHI3cbNPwjNPwUIE3sYlmMI30eTutzIvu0PnJs3uttaW4lgW==";
    private static String COOKIE = "";
    //private static final String A1 = "193d910b678dqc3ws7mk6hai96ojgzhr27f3ddtbd50000418268";

    private static  String B1 = "";
    private static String A1 = "1936108ad9bcw9vid6fgm3gndytwknc7j4eql9bg050000261389";
    //    private static final String B2 = "I38rHdgsjopgIvesdVwgIC+oIELmBZ5e3VwXLgFTIxS3bqwErFeexd0ekncAzMFYnqthIhJeDnMDKutRI3KsYorWHPtGrbV0P9WfIi/eWc6eYqtyQApPI37ekmR6QL+5Ii6sdnoeSfqYHqwl2qt5B0DoIx++GDi/sVtkIxdsxuwr4qtiIhuaIE3e3LV0I3VTIC7e0utl2ADmsLveDSKsSPw5IEvsiVtJOqw8BuwfPpdeTFWOIx4TIiu6ZPwrPut5IvlaLbgs3qtxIxes1VwHIkumIkIyejgsY/WTge7eSqte/D7sDcpipedeYrDtIC6eDVw2IENsSqtlnlSuNjVtIx5e1qt3bmAeVn8LIES/IhEe+AFDI3EPKI8BIiW7ZPwFIvGj4sesYINsxVwSIC7eSnde0fh+Ii3e6WrS8qwUIE7s1f0s6WAeiVtwpjNeWVwRIv48zA0efuwugfOsWVw8IxI2I38isqwZgVtPzg8QwcNejd/eiqwoIhAsS/AskFRYIk/s0MvskdE0IhgsiVwDIhGdQqwJ8ut9I33e3PtVIiNsiqwlIh/eDqtAHPwPmVwDI3MdIv4pH9ztrY3s3qwEIiT+IiesfPwoeWccpj3sDskuIkGyGuwbmPwhICdekVtUQpdeipJsTrELIhvs6m3ejPtsoI==";
//    private static final String B1 = "I38rHdgsjopgIvesdVwgIC+oIELmBZ5e3VwXLgFTIxS3bqwErFeexd0ekncAzMFYnqthIhJeSBMDKutRI3KsYorWHPtGrbV0P9WfIi/eWc6eYqtyQApPI37ekmR6QL+5Ii6sdneeSfqYHqwl2qt5B0DBIx++GDi/sVtkIxdsxuwr4qtiIhuaIE3e3LV0I3VTIC7e0utl2ADmsLveDSKsSPw5IEvsiVtJOqw8BuwfPpdeTFWOIx4TIiu6ZPwrPut5IvlaLbgs3qtxIxes1VwHIkumIkIyejgsY/WTge7eSqte/D7sDcpipedeYrDtIC6eDVw2IENsSqtlnlSuNjVtIx5e1qt3bmAeVn8LIESzIhEe+AFDI3EPKI8BIiW7ZPwFIvGj4sesYINsxVwSIC7eSnde0fh+Ii3e6WrS8qwUIE7s1f0s6WAeiVtwpjNeWVwRIv4naAAefqwUp9KsWVw8IxI2I38isqwZgVtPzg8QwcNejd/eiqwoIhAsS/AskFRYIk/s0MvskdE0IhgsiVwDIhGdQqwJ8ut9I33e3PtVIiNsVVwxIEgsfVtFHqwGmPw7IvrXIxIc29ufKZgexVthIhFcIi6eDVwcbYprLj7sWa+TIiIZ2qtPmVwuIvvekPtu+DNeVm7sVDzZIhAs37JsiPtgpVwaIvge0uwEIi/eT/LZIv7eS9oeSqt5Ix3sxuwU4eQtIED+Iv7sxINsxAgs0c7sdPwiIvHWIvIFcPwZICIRIhgsVL4lNjJsfqttIhp+mPtR4I0sdFoeYPwbeVwwICT7IkimpPwtKVtbIxgefS6sVPtprVw4IiPccuwdICpQp9MbIheedj6e007sVutJKVwyIhbgI3Je3uwuIkesWmTGIC8rrI+KIvRFIvJeVqwTguty+qwuICmAIE3sV0eeVPwVzphDICkswVwBLPw8IvbMICWoIEuRIkvs1losxnWXHW==";
    private static final String SEARCH_ID = "2e3kybh03tqry2p7yztgo";
    static String cursor = "";



    static Random rand = new Random();
    public static  void main(String[] args) throws Exception {
//        XiaohongshuCrawler.getKeywordList();
//        XiaohongshuCrawler.getAdvList();

        // 1. 获取配置
        Config config = ConfigService.getAppConfig(); // 获取当前应用的默认配置

        // 2. 获取配置项
        COOKIE = config.getProperty("xhs_cookie", "default_value");
        B1 = config.getProperty("xhs_b1", "default_value");
        A1 = config.getProperty("xhs_a1", "default_value");
        String xhsKeyword = config.getProperty("xhs_keyword", "default_value");
        String xhsCursor = config.getProperty("xhs_cursor", "default_value");
        String xhsUserId = config.getProperty("xhs_user_id", "default_value");
        System.out.println("Initial config value: " + COOKIE);

        // 3. 配置变更监听，监听配置变化
        config.addChangeListener(new ConfigChangeListener() {
            @Override
            public void onChange(ConfigChangeEvent changeEvent) {
                for (String changedKey : changeEvent.changedKeys()) {
                    ConfigChange change = changeEvent.getChange(changedKey);

                    switch(changedKey){
                        case "xhs_cookie":
                            COOKIE = change.getNewValue();
                            break;
                        case "xhs_b1":
                            B1 = change.getNewValue();
                            break;
                        case "xhs_a1":
                            A1 = change.getNewValue();
                            break;
                        default:
                            System.out.println("no keyvalue change!");
                    }
                    System.out.println("Config changed: " + changedKey + ", old value: " + change.getOldValue() + ", new value: " + change.getNewValue());
                }
            }
        });

        // 模拟等待一段时间，以接收配置更新
//        try {
//            Thread.sleep(60000);  // 等待一段时间以接收配置变化
//        } catch (InterruptedException e) {
//            e.printStackTrace();
//        }
        String[] arr = new String[]{"北京爱帝宫月子中心","深圳爱帝宫月子中心","成都爱帝宫月子会所","深圳爱帝宫月子会所","成都爱帝宫月子中心","北京爱帝宫月子会所","宝生月子会所","和伊母婴护理中心","贵州天恩母婴月子中心","禧月阁月子中心","上海爱尚家月子中心","积木月子中心","玥紫宫元月子中心"};
        String[] deparr = new String[]{"月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心"};


        List<Map<String,String>> feedList = new ArrayList<>();

        List<String[]> titleAliasList = new ArrayList<>();
        titleAliasList.add(new String[]{"keyword", "关键词"});
        titleAliasList.add(new String[]{"department", "科室"});
        titleAliasList.add(new String[]{"url", "FeedUrl"});
        titleAliasList.add(new String[]{"title", "文章标题"});
        titleAliasList.add(new String[]{"desc", "文案描述"});
        titleAliasList.add(new String[]{"likedCount", "点赞数"});
        titleAliasList.add(new String[]{"collectedCount", "收藏数"});
        titleAliasList.add(new String[]{"commentCount", "评论数"});
        titleAliasList.add(new String[]{"shareCount", "分享数"});
        for (int n = 0; n < arr.length; n++) {
            if(arr[n].equals(xhsKeyword)){
                ArrayList<HashMap<String, String>> noteList = getNoteList(xhsUserId, xhsCursor, new ArrayList<>());
                for (int j = 0; j < noteList.size(); j++) {
                    HashMap noteMap = noteList.get(j);
                    HashMap<String, String> feedMap = getFeedInfo(noteMap.get("noteId").toString(), noteMap.get("xsecToken").toString(), arr[n], deparr[n], xhsUserId, "keyword");

                    if (feedMap != null) {
                        feedList.add(feedMap);
                    }
                }
            }else {
                ArrayList<String> al = getUserBox(arr[n]);

                for (int i = 0; i < al.size(); i++) {
                    ArrayList<HashMap<String, String>> noteUserList = getUserPorfile(al.get(i));
                    ArrayList<HashMap<String, String>> noteList = getNoteList(al.get(i), XiaohongshuCrawler.cursor, noteUserList);
                    for (int j = 0; j < noteList.size(); j++) {
                        HashMap noteMap = noteList.get(j);
                        HashMap<String, String> feedMap = getFeedInfo(noteMap.get("noteId").toString(), noteMap.get("xsecToken").toString(), arr[n], deparr[n], al.get(i), "keyword");

                        if (feedMap != null) {
                            feedList.add(feedMap);
                        }
                    }
                }
            }
            ExcelUtils2.exportLocal(titleAliasList, feedList, "xiaohongshu", "小红书数据", arr[n] + ".xlsx", null);
            generalSql(feedList,arr[n]+".sql","0");
            feedList.clear();
            Thread.sleep(1000*60*5);
        }





//
//        Thread KeywordThread = new Thread(new KeywordTask());
////        Thread AdvThread = new Thread(new AdvTask());
//
//        KeywordThread.start();
////        AdvThread.start();

    }
//    static class KeywordTask implements Runnable {
//        @Override
//        public void run() {
//            try {
//                getKeywordList();
//            } catch (Exception e) {
//                throw new RuntimeException(e);
//            }
//        }
//    }
//
//
//    class AdvTask implements Runnable {
//        @Override
//        public void run() {
//            try {
//                getAdvList();
//            } catch (Exception e) {
//                throw new RuntimeException(e);
//            }
//        }
//    }

    public static ArrayList<HashMap<String,String>> getUserPorfile(String userId) throws IOException {
        ArrayList<HashMap<String,String>> userNoteList = new ArrayList<>();
        OkHttpClient client = new OkHttpClient().newBuilder()
                .build();
        MediaType mediaType = MediaType.parse("text/plain");
        RequestBody body = RequestBody.create(mediaType, "");
        Request request = new Request.Builder()
                .url("https://www.xiaohongshu.com/user/profile/"+userId)
                .method("GET", null)
                .addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
                .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
                .addHeader("cache-control", "max-age=0")
                .addHeader("cookie", COOKIE)
                .addHeader("priority", "u=0, i")
                .addHeader("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"")
                .addHeader("sec-ch-ua-mobile", "?0")
                .addHeader("sec-ch-ua-platform", "\"Windows\"")
                .addHeader("sec-fetch-dest", "document")
                .addHeader("sec-fetch-mode", "navigate")
                .addHeader("sec-fetch-site", "none")
                .addHeader("sec-fetch-user", "?1")
                .addHeader("upgrade-insecure-requests", "1")
                .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
                .build();
        Response response = client.newCall(request).execute();
        String html = response.body().string();
        String jsonString = html.substring(html.lastIndexOf("__INITIAL_STATE__")+18,html.lastIndexOf("</script>"));
        if(jsonString != null) {
            JSONObject userObj = JSON.parseObject(jsonString).getJSONObject("user");
            JSONArray notes = userObj.getJSONArray("notes").getJSONArray(0);
            for (int i = 0; i < notes.size(); i++) {
                HashMap<String,String> noteMap = new HashMap<>();
                JSONObject noteObj = notes.getJSONObject(i);
                String noteId = noteObj.getString("id");
                String xsecToken = noteObj.getString("xsecToken");

                noteMap.put("noteId", noteId);
                noteMap.put("xsecToken", xsecToken);
                userNoteList.add(noteMap);
            }
            JSONObject noteQueriesObj = userObj.getJSONArray("noteQueries").getJSONObject(0);
            XiaohongshuCrawler.cursor = noteQueriesObj.getString("cursor");
        }
        return  userNoteList;

    }

    public static ArrayList<String> getUserBox(String keyword) throws Exception {
        ArrayList<String> al = new ArrayList<>();
        OkHttpClient client = new OkHttpClient().newBuilder()
                .build();
        MediaType mediaType = MediaType.parse("application/json;charset=UTF-8");
        String paramJsonStr = "{\"url\": \"/api/sns/web/v1/search/onebox\",\"b1\": \"" + B1 + "\",\"cookieParam\":\""+COOKIE+"\",\"a1\":\""+A1+"\",\"data\":\"{\\\"keyword\\\":\\\""+keyword+"\\\",\\\"search_id\\\":\\\"2e6u9zy4rbrvva51covnp\\\",\\\"biz_type\\\":\\\"web_search_user\\\",\\\"request_id\\\":\\\"1318420511-1734941436070\\\"}\"}";
        Map<String, String> xsMap = XiaohongshuCrawler.getNoteXs(paramJsonStr,"keyword");
        if (xsMap.isEmpty() && xsMap.get("xs") != null) {
            throw new Exception("获取xsMap失败");
        }
        RequestBody body = RequestBody.create(mediaType, "{\"keyword\":\""+keyword+"\",\"search_id\":\"2e6u9zy4rbrvva51covnp\",\"biz_type\":\"web_search_user\",\"request_id\":\"1318420511-1734941436070\"}");
        Request request = new Request.Builder()
                .url("https://edith.xiaohongshu.com/api/sns/web/v1/search/onebox")
                .method("POST", body)
                .addHeader("accept", "application/json, text/plain, */*")
                .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
                .addHeader("content-type", "application/json;charset=UTF-8")
                .addHeader("cookie", COOKIE)
                .addHeader("origin", "https://www.xiaohongshu.com")
                .addHeader("priority", "u=1, i")
                .addHeader("referer", "https://www.xiaohongshu.com/")
                .addHeader("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"")
                .addHeader("sec-ch-ua-mobile", "?0")
                .addHeader("sec-ch-ua-platform", "\"Windows\"")
                .addHeader("sec-fetch-dest", "empty")
                .addHeader("sec-fetch-mode", "cors")
                .addHeader("sec-fetch-site", "same-site")
                .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
                .addHeader("x-b3-traceid", xsMap.get("b3Traceid").toString())
                .addHeader("x-xray-traceid", xsMap.get("xrayTraceid").toString())
                .addHeader("x-s", xsMap.get("xs").toString())
                .addHeader("x-s-common", xsMap.get("xsCommon").toString())
                .addHeader("x-t", xsMap.get("xt").toString())
                .build();
        Response response = client.newCall(request).execute();
        String rsponseStr = response.body().string();
//        System.out.println("feed==:"+rsponseStr);
        JSONObject obj = JSON.parseObject(rsponseStr);
        String code = obj.getString("code");
        if(!"0".equals(code)){
            System.out.println("UserBox==:"+rsponseStr);
            return  null;
        }
        JSONArray oneboxList = obj.getJSONObject("data").getJSONArray("onebox_list");

        if (oneboxList == null){
            System.out.println("账号‘"+keyword+"'找不到数据");
            return al;
        }
        if(!oneboxList.isEmpty()){
            for (int i = 0; i < oneboxList.size(); i++) {
                String  userId = oneboxList.getJSONObject(i).getJSONObject("user_one_box").getString("id");
                al.add(userId);
            }
        }
        return al;
    }

    public static ArrayList<HashMap<String,String>> getNoteList(String userId,String cursor,ArrayList<HashMap<String,String>> noteUserList) throws Exception {

        ArrayList<HashMap<String,String>> noteList = new ArrayList<>();
        noteList.addAll(noteUserList);
        OkHttpClient client = new OkHttpClient().newBuilder()
                .build();
        MediaType mediaType = MediaType.parse("text/plain");
        boolean hasMore = true;
        if("".equals(cursor)){
            return noteList;
        }

        while (hasMore) {

            String paramJsonStr = "{\"url\": \"/api/sns/web/v1/user_posted?num=30&cursor="+cursor+"&user_id=" + userId + "&image_formats=jpg,webp,avif&xsec_token=&xsec_source=\",\"b1\": \"" + B1 + "\",\"cookieParam\":\""+COOKIE+"\",\"a1\":\""+A1+"\",\"data\":\"null\"}";
            Map<String, String> xsMap = XiaohongshuCrawler.getNoteXs(paramJsonStr, "keyword");
            if (xsMap.isEmpty() && xsMap.get("xs") != null) {
                throw new Exception("获取xsMap失败");
            }
//            RequestBody body = RequestBody.create(mediaType, "");
            Request request = new Request.Builder()
                    .url("https://edith.xiaohongshu.com/api/sns/web/v1/user_posted?num=30&cursor="+cursor+"&user_id=" + userId + "&image_formats=jpg,webp,avif&xsec_token=&xsec_source=")
                    .method("GET", null)
                    .addHeader("accept", "application/json, text/plain, */*")
                    .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
                    .addHeader("cookie", COOKIE)
                    .addHeader("origin", "https://www.xiaohongshu.com")
                    .addHeader("priority", "u=1, i")
                    .addHeader("referer", "https://www.xiaohongshu.com/")
                    .addHeader("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"")
                    .addHeader("sec-ch-ua-mobile", "?0")
                    .addHeader("sec-ch-ua-platform", "\"Windows\"")
                    .addHeader("sec-fetch-dest", "empty")
                    .addHeader("sec-fetch-mode", "cors")
                    .addHeader("sec-fetch-site", "same-site")
                    .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
                    .addHeader("x-b3-traceid", xsMap.get("b3Traceid").toString())
                    .addHeader("x-xray-traceid", xsMap.get("xrayTraceid").toString())
                    .addHeader("x-s", xsMap.get("xs").toString())
                    .addHeader("x-s-common", xsMap.get("xsCommon").toString())
                    .addHeader("x-t", xsMap.get("xt").toString())
                    .build();
            Response response = client.newCall(request).execute();
            String rsponseStr = response.body().string();
//        System.out.println("feed==:"+rsponseStr);
            JSONObject obj = JSON.parseObject(rsponseStr);
            String code = obj.getString("code");
            if (!"0".equals(code)) {
                System.out.println("getNoteList:" + rsponseStr);
                return null;
            }
            JSONObject dataObj = obj.getJSONObject("data");
            JSONArray notesArr = dataObj.getJSONArray("notes");
            hasMore = dataObj.getBoolean("has_more");
            cursor = dataObj.getString("cursor");
            XiaohongshuCrawler.cursor = cursor;
            for (int i = 0; i < notesArr.size(); i++) {
                JSONObject noteObj = notesArr.getJSONObject(i);
                String noteId = noteObj.getString("note_id");
                String xsecToken = noteObj.getString("xsec_token");
                HashMap<String, String> noteMap = new HashMap<>();
                noteMap.put("noteId", noteId);
                noteMap.put("xsecToken", xsecToken);
                noteList.add(noteMap);
            }
            Thread.sleep((rand.nextInt(3))*1000);

        }
        return noteList;
    }
    public static HashMap<String,String> getFeedInfo(String id, String xsecToken,String keyword,String department,String userId,String accountType) throws Exception {

        StringBuffer descSb = new StringBuffer();
        String paramJsonStr = "{\"url\": \"/api/sns/web/v1/feed\",\"b1\": \""+B1+"\",\"cookieParam\":\""+COOKIE+"\",\"a1\":\""+A1+"\",\"data\":\"{\\\"source_note_id\\\":\\\""+id+"\\\",\\\"image_formats\\\":[\\\"jpg\\\",\\\"webp\\\",\\\"avif\\\"],\\\"extra\\\":{\\\"need_body_topic\\\":\\\"1\\\"},\\\"xsec_source\\\":\\\"pc_user\\\",\\\"xsec_token\\\":\\\""+xsecToken+"\\\"}\"}";
        Map<String,String> xsMap = XiaohongshuCrawler.getNoteXs(paramJsonStr,accountType);
        if (xsMap.isEmpty()||xsMap==null){
            throw new Exception("获取xsMap失败");
        }
        OkHttpClient client = new OkHttpClient().newBuilder()
                // 设置连接超时为 60 秒
                .connectTimeout(60, TimeUnit.SECONDS)
                // 设置读取超时为 60 秒
                .readTimeout(60, TimeUnit.SECONDS)
                // 设置写入超时为 60 秒
                .writeTimeout(10, TimeUnit.SECONDS)
                .build();
        MediaType mediaType = MediaType.parse("application/json;charset=UTF-8");
        RequestBody body = RequestBody.create(mediaType, "{\"source_note_id\":\""+id+"\",\"image_formats\":[\"jpg\",\"webp\",\"avif\"],\"extra\":{\"need_body_topic\":\"1\"},\"xsec_source\":\"pc_user\",\"xsec_token\":\""+xsecToken+"\"}");
        Request request = new Request.Builder()
                .url("https://edith.xiaohongshu.com/api/sns/web/v1/feed")
                .method("POST", body)
                .addHeader("accept", "application/json, text/plain, */*")
                .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
                .addHeader("cache-control", "no-cache")
                .addHeader("content-type", "application/json;charset=UTF-8")
                .addHeader("cookie", COOKIE)
                .addHeader("origin", "https://www.xiaohongshu.com")
                .addHeader("pragma", "no-cache")
                .addHeader("priority", "u=1, i")
                .addHeader("referer", "https://www.xiaohongshu.com/")
                .addHeader("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"")
                .addHeader("sec-ch-ua-mobile", "?0")
                .addHeader("sec-ch-ua-platform", "\"Windows\"")
                .addHeader("sec-fetch-dest", "empty")
                .addHeader("sec-fetch-mode", "cors")
                .addHeader("sec-fetch-site", "same-site")
                .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
                .addHeader("x-b3-traceid", xsMap.get("b3Traceid").toString())
                .addHeader("x-xray-traceid", xsMap.get("xrayTraceid").toString())
                .addHeader("x-s", xsMap.get("xs").toString())
                .addHeader("x-s-common", xsMap.get("xsCommon").toString())
                .addHeader("x-t", xsMap.get("xt").toString())
                .build();


        Response response = client.newCall(request).execute();
        String rsponseStr = response.body().string();
//        System.out.println("feed==:"+rsponseStr);
        JSONObject obj = JSON.parseObject(rsponseStr);
        String code = obj.getString("code");
        if("300013".equals(code)||"-100".equals(code)){
            Thread.sleep(60*1000*5);
            System.out.println("code:"+code+",keyword:"+keyword+",id:"+id+",xsecToken:"+xsecToken+",cursor:"+XiaohongshuCrawler.cursor);
            getFeedInfo(id, xsecToken,keyword,department,userId,accountType);
        }
        if(!"0".equals(code)){
            System.out.println("feed未知错误:"+rsponseStr);
            log.error("code:{},keyword:{},id:{},xsecToken:{}",code,keyword,id,xsecToken);
            System.out.println("code:"+code+",keyword:"+keyword+",id:"+id+",xsecToken:"+xsecToken+",cursor:"+XiaohongshuCrawler.cursor);
            Thread.sleep(60*1000*5);
        }else{
            Thread.sleep((rand.nextInt(4)+10)*1000);
        }
        JSONArray dataJsonArr = obj.getJSONObject("data").getJSONArray("items");
        JSONObject itemObj = JSON.parseObject(dataJsonArr.get(0).toString()).getJSONObject("note_card");
        JSONObject interactInfo = itemObj.getJSONObject("interact_info");
        String likedCount = interactInfo.getString("liked_count");
        String collectedCount = interactInfo.getString("collected_count");
        String commentCount = interactInfo.getString("comment_count");
        String shareCount = interactInfo.getString("share_count");
        String desc = itemObj.getString("desc");
        String title = itemObj.getString("title");
        String dateTime = itemObj.getString("time");
        String noteId  = itemObj.getString("note_id");
        descSb.setLength(0);
        String url = descSb.append("https://www.xiaohongshu.com/explore/").append(id).append("?xsec_token=").append(xsecToken).append("&xsec_source=pc_search&source=web_explore_feed").toString();
        HashMap<String,String> resultMap = new HashMap<>();
        resultMap.put("url",url);
        resultMap.put("keyword",keyword);
        resultMap.put("department",department);
        resultMap.put("desc",desc);
        resultMap.put("title",title);
        resultMap.put("noteId",noteId);
        resultMap.put("userId",userId);
        resultMap.put("likedCount",convertCount(likedCount));
        resultMap.put("collectedCount",convertCount(collectedCount));
        resultMap.put("commentCount",convertCount(commentCount));
        resultMap.put("shareCount",convertCount(shareCount));
        resultMap.put("xhsCreateTime",convertTimeStamp(dateTime));
        return  resultMap;
    }
    public static Map<String, String> getXs(String paramJsonStr,String accountType) throws  IOException {
        String url = "http://10.200.1.100:8001/post_signatures/x1";
        if("adv".equals(accountType)){
            url = "http://10.200.1.100:8001/post_signatures/x1";
        }
        OkHttpClient client = new OkHttpClient().newBuilder()
                .build();
        MediaType mediaType = MediaType.parse("application/json");
        RequestBody body = RequestBody.create(mediaType, paramJsonStr);
        Request request = new Request.Builder()
                .url(url)
                .method("POST", body)
                .addHeader("Content-Type", "application/json")
                .build();
        Response response = client.newCall(request).execute();

        String rsponseStr = response.body().string();
//        System.out.println(rsponseStr);
        JSONObject obj = JSON.parseObject(rsponseStr);
        String xs = obj.getString("X-s");
        String xt = obj.getString("X-t");
        String xsCommon = obj.getString("xs_common");
//        System.out.println("xs:"+xs);
//        System.out.println("xt:"+xt);
//        System.out.println("xsCommon:"+xsCommon);
        Map<String,String> xsMap = new HashMap<>();
        xsMap.put("xs",xs);
        xsMap.put("xt",xt);
        xsMap.put("xsCommon",xsCommon);
        return xsMap;
    }

    public static Map<String, String> getNoteXs(String paramJsonStr,String accountType) throws  IOException {
        String url = "http://10.210.2.231:8081/excute/xs";
        OkHttpClient client = new OkHttpClient().newBuilder()
                .build();
        MediaType mediaType = MediaType.parse("application/json");
        RequestBody body = RequestBody.create(mediaType, paramJsonStr);
        Request request = new Request.Builder()
                .url(url)
                .method("POST", body)
                .addHeader("Content-Type", "application/json")
                .build();
        Response response = client.newCall(request).execute();

        String rsponseStr = response.body().string();
//        System.out.println(rsponseStr);
        JSONObject obj = JSON.parseObject(rsponseStr);
        String xs = obj.getString("xs");
        String xt = obj.getString("xt");
        String xsCommon = obj.getString("xsCommon");
        String xrayTraceid = obj.getString("xrayTraceid");
        String b3Traceid = obj.getString("b3Traceid");
//        System.out.println("xs:"+xs);
//        System.out.println("xt:"+xt);
//        System.out.println("xsCommon:"+xsCommon);
        Map<String,String> xsMap = new HashMap<>();
        xsMap.put("xs",xs);
        xsMap.put("xt",xt);
        xsMap.put("xsCommon",xsCommon);
        xsMap.put("xrayTraceid",xrayTraceid);
        xsMap.put("b3Traceid",b3Traceid);
        return xsMap;
    }


    public static void getAdvList() throws Exception {

        OkHttpClient client = new OkHttpClient().newBuilder()
                // 设置连接超时为 60 秒
                .connectTimeout(60, TimeUnit.SECONDS)
                // 设置读取超时为 60 秒
                .readTimeout(60, TimeUnit.SECONDS)
                // 设置写入超时为 60 秒
                .writeTimeout(10, TimeUnit.SECONDS)
                .build();
        MediaType mediaType = MediaType.parse("application/json;charset=UTF-8");

        List<Map<String,String>> feedList = new ArrayList<>();

        List<String[]> titleAliasList = new ArrayList<>();
        titleAliasList.add(new String[]{"keyword", "关键词"});
        titleAliasList.add(new String[]{"department", "科室"});
        titleAliasList.add(new String[]{"url", "FeedUrl"});
        titleAliasList.add(new String[]{"title", "文章标题"});
        titleAliasList.add(new String[]{"desc", "文案描述"});
        titleAliasList.add(new String[]{"likedCount", "点赞数"});
        titleAliasList.add(new String[]{"collectedCount", "收藏数"});
        titleAliasList.add(new String[]{"commentCount", "评论数"});
        titleAliasList.add(new String[]{"shareCount", "分享数"});


        HashMap<String,String> departmentMap = new HashMap<>();
//        departmentMap.put("eys","眼科医疗");
//        departmentMap.put("physical","体检机构(新)");
//        departmentMap.put("mas","医美外科(新)");
        departmentMap.put("mb","医疗美容");
//        departmentMap.put("pog","月子妇产(新)");
//        departmentMap.put("htn","植发养发"); //已完成
//        departmentMap.put("om","口腔医疗");
        HashMap<String,List> advMap = new HashMap<>();
        for (Map.Entry<String, String> entry : departmentMap.entrySet()) {
            String departmentKey = entry.getKey();
            String department = entry.getValue();
            List<Map<String,String>> advList = new ArrayList<>();
            int i = 323;
            while (true){
                i++;


                String paramJsonStr = "{\"url\": \"/api/partner/industry_plate/note_detail_list\",\"b1\": \""+B1+"\",\"data\":\"{\\\"viewType\\\": 40,\\\"dimensionType\\\": 44,\\\"frontFilterList\\\": [{\\\"filterType\\\": \\\"52\\\",\\\"filterField\\\": \\\"commercial_taxonomy_name1|commercial_taxonomy_name2\\\",\\\"filterValues\\\": [\\\"医疗医美|"+department+"\\\"]},{\\\"filterType\\\": \\\"4\\\",\\\"filterField\\\": \\\"date_key\\\",\\\"filterValues\\\": [\\\"2024-11-06\\\",\\\"2024-12-06\\\"]}],\\\"dynamicTargets\\\": [\\\"title\\\",\\\"nickname\\\",\\\"note_url\\\",\\\"image_url\\\",\\\"is_self\\\",\\\"customer_level\\\",\\\"effective_account\\\",\\\"imp_num\\\",\\\"click_num\\\",\\\"interaction\\\",\\\"ctr\\\",\\\"cpc\\\",\\\"cpm\\\",\\\"cpe\\\",\\\"interaction_cvr\\\",\\\"enter_seller_7d_cpl\\\",\\\"enter_seller_7d_ctr\\\",\\\"goods_view_7d_cpl\\\",\\\"goods_view_7d_ctr\\\",\\\"add_cart_7d_cpl\\\",\\\"add_cart_7d_ctr\\\",\\\"total_order_num_7d_cpl\\\",\\\"total_order_num_7d_cvr\\\",\\\"total_order_gmv_7d_roi\\\",\\\"purchase_order_num_7d_cvr\\\",\\\"purchase_order_num_7d_cpl\\\",\\\"purchase_order_gmv_7d_roi\\\",\\\"purchase_order_gmv_30d_roi\\\",\\\"cpl\\\",\\\"leads_cvr\\\",\\\"message_open_cpl\\\",\\\"message_open_cvr\\\",\\\"message_driving_open_cpl\\\",\\\"message_driving_open_cvr\\\",\\\"msg_leads_num_cpl\\\",\\\"msg_leads_num_cvr\\\",\\\"entry_live_pv_cpl\\\",\\\"live_24h_click_effective_shutdown_num_cpl\\\",\\\"live_purchase_order_num_24h_cpl\\\",\\\"live_purchase_order_gmv_24h_roi\\\",\\\"clk_live_avg_view_time\\\",\\\"out_click_goods_view_pv_7d_cpl\\\",\\\"out_click_goods_view_pv_7d_cvr\\\",\\\"out_click_total_order_7d_cvr\\\",\\\"out_click_total_order_7d_cpl\\\",\\\"out_click_7d_roi\\\",\\\"out_click_total_order_15d_cvr\\\",\\\"out_click_total_order_15d_cpl\\\",\\\"out_click_15d_roi\\\",\\\"out_click_total_order_30d_cvr\\\",\\\"out_click_total_order_30d_cpl\\\",\\\"out_click_30d_roi\\\",\\\"ctr\\\",\\\"note_type\\\"],\\\"compFrontFilterList\\\": [],\\\"sortField\\\": \\\"ctr\\\",\\\"sortType\\\": \\\"desc\\\",\\\"pageIndex\\\": "+i+",\\\"pageSize\\\": 10,\\\"source\\\": \\\"partner\\\"}\"}";
                Map<String,String> xsMap = XiaohongshuCrawler.getXs(paramJsonStr,"adv");
                if (xsMap.isEmpty()){
                    throw new Exception("获取xsMap失败");
                }

                RequestBody body = RequestBody.create(mediaType, "{\"viewType\": 40,\"dimensionType\": 44,\"frontFilterList\": [{\"filterType\": \"52\",\"filterField\": \"commercial_taxonomy_name1|commercial_taxonomy_name2\",\"filterValues\": [\"医疗医美|"+department+"\"]},{\"filterType\": \"4\",\"filterField\": \"date_key\",\"filterValues\": [\"2024-11-06\",\"2024-12-06\"]}],\"dynamicTargets\": [\"title\",\"nickname\",\"note_url\",\"image_url\",\"is_self\",\"customer_level\",\"effective_account\",\"imp_num\",\"click_num\",\"interaction\",\"ctr\",\"cpc\",\"cpm\",\"cpe\",\"interaction_cvr\",\"enter_seller_7d_cpl\",\"enter_seller_7d_ctr\",\"goods_view_7d_cpl\",\"goods_view_7d_ctr\",\"add_cart_7d_cpl\",\"add_cart_7d_ctr\",\"total_order_num_7d_cpl\",\"total_order_num_7d_cvr\",\"total_order_gmv_7d_roi\",\"purchase_order_num_7d_cvr\",\"purchase_order_num_7d_cpl\",\"purchase_order_gmv_7d_roi\",\"purchase_order_gmv_30d_roi\",\"cpl\",\"leads_cvr\",\"message_open_cpl\",\"message_open_cvr\",\"message_driving_open_cpl\",\"message_driving_open_cvr\",\"msg_leads_num_cpl\",\"msg_leads_num_cvr\",\"entry_live_pv_cpl\",\"live_24h_click_effective_shutdown_num_cpl\",\"live_purchase_order_num_24h_cpl\",\"live_purchase_order_gmv_24h_roi\",\"clk_live_avg_view_time\",\"out_click_goods_view_pv_7d_cpl\",\"out_click_goods_view_pv_7d_cvr\",\"out_click_total_order_7d_cvr\",\"out_click_total_order_7d_cpl\",\"out_click_7d_roi\",\"out_click_total_order_15d_cvr\",\"out_click_total_order_15d_cpl\",\"out_click_15d_roi\",\"out_click_total_order_30d_cvr\",\"out_click_total_order_30d_cpl\",\"out_click_30d_roi\",\"ctr\",\"note_type\"],\"compFrontFilterList\": [],\"sortField\": \"ctr\",\"sortType\": \"desc\",\"pageIndex\": "+i+",\"pageSize\": 10,\"source\": \"partner\"}");
                Request request = new Request.Builder()
                        .url("https://partner.xiaohongshu.com/api/partner/industry_plate/note_detail_list")
                        .method("POST", body)
                        .addHeader("accept", "application/json, text/plain, */*")
                        .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
                        .addHeader("content-type", "application/json;charset=UTF-8")
                        .addHeader("cookie", "abRequestId=4688e422-3bd2-589f-b891-fe4fcd33a55f; a1=1936108ad9bcw9vid6fgm3gndytwknc7j4eql9bg050000261389; webId=29dad20020f97cab8cafc205030d714b; gid=yjqKy8YDWjViyjqKy8Y0f4jDjDSEjh3fKik6dk8qkIfv9A28ETISW1888JKyqYj88JqK2DjD; web_session=040069b1ad2a33d0354c5eed73354be54667cc; x-user-id-partner.xiaohongshu.com=67496db54f00000000000002; customerClientId=826692150613376; webBuild=4.46.0; unread={%22ub%22:%2267516ff9000000000203ab9a%22%2C%22ue%22:%22674eef0e00000000070316d2%22%2C%22uc%22:25}; xsecappid=partner-shell; websectiga=3633fe24d49c7dd0eb923edc8205740f10fdb18b25d424d2a2322c6196d2a4ad; sec_poison_id=b1df1d51-38c7-43c0-a564-d76af50f7a47; acw_tc=0a0d09d017337095928693455edbcd2953eb034bf409b55a5fade22e32b2c4; customer-sso-sid=68c517446226002789063427405d9d81dca677f6; ads-partner-sid=5caf1f3b54e14d02980ad66bb29f4ef2; system.alias=ads_partner; access-token-partner.xiaohongshu.com=customer.ads_partner.AT-68c517446226007084030755tfppsfze3kptxdov")
                        .addHeader("origin", "https://partner.xiaohongshu.com")
                        .addHeader("priority", "u=1, i")
                        .addHeader("referer", "https://partner.xiaohongshu.com/partner/industry-dashboard")
                        .addHeader("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"")
                        .addHeader("sec-ch-ua-mobile", "?0")
                        .addHeader("sec-ch-ua-platform", "\"Windows\"")
                        .addHeader("sec-fetch-dest", "empty")
                        .addHeader("sec-fetch-mode", "cors")
                        .addHeader("sec-fetch-site", "same-origin")
                        .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
                        .addHeader("x-b3-traceid", "dde1d6f3376fe26f")
                        .addHeader("x-s", xsMap.get("xs").toString())
                        .addHeader("x-t", xsMap.get("xt").toString())
                        .build();
                Response response = client.newCall(request).execute();

                String rsponseStr = response.body().string();
    //            System.out.println("feed==:"+rsponseStr);

                JSONObject obj = JSON.parseObject(rsponseStr);
                if(obj==null){
                    System.out.println("adv==:"+rsponseStr);
                    continue;
                }
                String code = obj.getString("code");
                if (!"0".equals(code)) {
                    System.out.println("adv==:"+rsponseStr);
                    continue;
                }
                JSONArray detailVoListArr = obj.getJSONObject("data").getJSONObject("detailListVo").getJSONArray("detailVoList");
                int pageCount = obj.getJSONObject("data").getJSONObject("detailListVo").getIntValue("pageCount");
                for (int j = 0; j < detailVoListArr.size(); j++) {
                    String url = detailVoListArr.getJSONObject(j).getJSONArray("targetList").getJSONObject(2).getString("targetValue");
                    String id = url.substring(url.indexOf("item/")+5,url.indexOf("?xsec_token="));
                    String xsecToken = url.substring(url.indexOf("xsec_token=")+11,url.indexOf("&xsec_source=pc_partner"));
                    HashMap<String,String> map = new HashMap<>();
                    map.put("id",id);
                    map.put("xsecToken",xsecToken);
                    advList.add(map);
                }

                System.out.println("i======================"+i);
                if(i == pageCount){
                    System.out.println("i=========="+department+"==========="+departmentKey);
                    advMap.put(departmentKey,advList);
                    break;
                }
            }
        }

        for (Map.Entry<String, List> entry : advMap.entrySet()) {
            String key = entry.getKey();
            List advList = entry.getValue();

            for (int j = 0; j < advList.size(); j++) {
                HashMap<String, String> advFeedMap = (HashMap<String, String>) advList.get(j);
                HashMap<String, String> feedMap = getFeedInfo(advFeedMap.get("id"), advFeedMap.get("xsecToken"),"医疗美容",departmentMap.get(key),"","adv");
                if (feedMap != null) {
                    feedList.add(feedMap);
                }

                if(feedList.size()%1000==0){
                    ExcelUtils2.exportLocal(titleAliasList, feedList, "xiaohongshu", "小红书数据", departmentMap.get(key)+new Date().getTime() + ".xlsx", null);
                    generalSql(feedList,departmentMap.get(key)+new Date().getTime() + ".sql","1");
                    feedList.clear();
                    Thread.sleep(1000*60*5);
                }

                if(j == (advList.size()-1)){
                    ExcelUtils2.exportLocal(titleAliasList, feedList, "xiaohongshu", "小红书数据", departmentMap.get(key)+new Date().getTime() + ".xlsx", null);
                    generalSql(feedList,departmentMap.get(key)+new Date().getTime() + ".sql","1");
                    feedList.clear();
                }
                System.out.println("j======================"+j);

            }
        }


    }

    public static void generalSql (List<Map<String,String>> rows,String fileName,String flag){
        final String preSql="INSERT INTO `business_report_data`.`xhs_keyword_record` (`keyword`,`xhs_account_id`, `note_id`, `department`, `feed_url`, `title`, `desc`, `liked_count`, `collected_count`, `comment_count`, `share_count`, `flag`, `xhs_create_time`) VALUES ('";
        StringBuffer sbSql = new StringBuffer();
        String filePath = "D://exportExcel//sql//"+fileName;  // 目标文件路径
        File file = new File(filePath);

        try {
            // 如果文件不存在则创建
            if (file.createNewFile()) {
                System.out.println("File created: " + file.getName());
            } else {
                System.out.println("File already exists.");
            }
        } catch (IOException e) {
            System.err.println("An error occurred while creating the file.");
            e.printStackTrace();
        }

        for (int i = 0; i < rows.size(); i++) {
            HashMap<String, String> map = (HashMap<String, String>) rows.get(i);
            sbSql.setLength(0);
            sbSql.append(preSql)
                    .append(map.get("keyword"))
                    .append("','")
                    .append(map.get("userId"))
                    .append("','")
                    .append(map.get("noteId"))
                    .append("','")
                    .append(map.get("department"))
                    .append("','")
                    .append(map.get("url"))
                    .append("','")
                    .append(map.get("title"))
                    .append("','")
                    .append(map.get("desc"))
                    .append("','")
                    .append(map.get("likedCount"))
                    .append("','")
                    .append(map.get("collectedCount"))
                    .append("','")
                    .append(map.get("commentCount"))
                    .append("','")
                    .append(map.get("shareCount"))
                    .append("','")
                    .append(flag)
                    .append("','")
                    .append(map.get("xhsCreateTime"))
                    .append("');");

            try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(filePath, true))) {
                bufferedWriter.write(sbSql.toString());
                bufferedWriter.newLine();  // 使用 newLine() 添加换行符
            } catch (IOException e) {
                System.err.println("Error occurred while appending to file: " + e.getMessage());
            }
        }
    }

    public static String convertCount(String countStr){
        int index = countStr.replaceAll("\\s+", "").indexOf("万");
        float b = 10000f;
        if(index>0){
            float num = Float.parseFloat(countStr.substring(0,index));
            return String.format("%.0f", num * b);
        }else {
            return countStr;
        }

    }

    public static  String convertTimeStamp(String timeStr){
        Long timestamp = Long.parseLong(timeStr);
        // 转换为 Instant
        Instant instant = Instant.ofEpochMilli(timestamp);

        // 转换为 LocalDateTime（本地时间）
        LocalDateTime dateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());

        // 使用 DateTimeFormatter 格式化 LocalDateTime
        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        String formattedDate = dateTime.format(formatter);

//        System.out.println("Formatted Date: " + formattedDate);  // 输出: Formatted Date: 2021-12-04 00:00:00
        return formattedDate;
    }

    public static void getKeywordList() throws Exception {
        //        String[] arr = new String[]{"徐鲁·雍禾植发","高新米兰柏羽","雍禾毛发中心","雍禾不剃发植发医院","广州曙光医学美容医院","雍禾植发医院","伊美尔","mylike漾格医美","四川华美紫馨医疗美容","雍禾毛发医院","碧莲盛植发全国连锁"};
        String[] arr = new String[]{"穗华口腔佛山院区","齐美矫正","重庆维乐口腔","成都瑞泰口腔","福州维乐种植牙口腔","牙博士口腔","中科领军口腔正骑中心","北京茗视光眼科","鹏爱科学抗衰","上海爱尔眼科医院（总院","杭州黑马眼科","贵阳麦迪格眼科","希玛眼科","济南普瑞眼科医院","南京维视眼科医院","成都睛彩眼料","成都高新睛彩眼科","上海华厦柠盟眼科","泽明眼科医院","辽宁爱尔眼科医院","广州美莱医疗美容","上海美莱","健丽医美专注眼袋","南京米兰柏羽","北京美莱医美","北京京城皮肤医院","深圳美莱","重庆华美","北京艺星整形","四川米兰柏羽总院","上海德琳","mylikc·漾格医美","米兰柏羽","广州曙光美肤中心","华美紫馨抗衰美肤中心","郑州美莱医疗美容","深圳阳光医美抗衰中心","成都圣贝口腔","佛官口腔","重庆维乐口腔医院","苏州美奥口腔门诊部","拜丽口腔美学中心","永华口腔","上海申洁口腔医院","杭州西湖口腔医院古地路","广州广大口腔","福州维乐口腔","北京整齐姓姓口腔医院","鱼鱼说口腔","福斯曼牙齿正路","军海口腔连锁","浙杭口腔","美美口腔MYOURDENT","爱牙无忧口腔医疗服务","医大口腔（华西派）","爱康健口腔医院","同步口腔","维乐口腔重庆","南宁微笑天使口腔","北京禾禾齿科","杭州牙科医院","杭州齐尔口腔官方账号","佛山曙光金子口腔","齐美矫正正畸中心","贵阳柏乐口腔","杭州齐尔口腔","成都极光口腔","诺恩口腔","登特儿童口腔医院","重庆瑞泰口腔","中诺第二口腔医院","武汉通策口腔医院","南京康贝佳口腔医院","XM思迈尔口腔","牙牙无优口腔咨询","泰皇岛德勤口腔","美奥口腔·长沙","VIC·Smile唯刻美口腔","上海正睿齿科","六月微笑口腔美学","曙康口腔","北京市劲松口腔医院","美优齐口腔","阜阳住德口腔医院","成都春天牙科","北京维尔口腔","免博士口腔","郑州唯美口腔医院","佛山爱顿口腔","成都锦官口腔","南医大友谊泰州分院","杭州科瓦齿科","沈阳米|兰口腔","美可普口腔","北京碧格口腔","北京劲松口腔医院","柠盟眼科","英华眼科官方账号","北京普瑞华德眼科","成都爱尔眼科医院","维视眼科医院","合肥瑶海普瑞眼科","黑马眼科医院","深圳博爱曙光医院眼科","福州爱尔眼科医院","深圳新安希玛林顺潮眼利","普瑞眼科总院","东莞光明眼科医院","重庆何氏眼科医院","EuroEyes德视佳眼科","福州黑马眼科","南昌普瑞眼科医院","昆明爱尔眼科","爱尔眼科上海","上海新视界眼科医院","英华眼科","尖峰眼科-郑州","郑州尖峰眼科","重庆爱尔眼科总院","深圳爱尔眼科医院","长沙爱尔眼科医院","南宁普瑞眼科医院","新视界眼科","EaroEyes上海德视佳眼利","北京惟视新视野眼科","视献眼科","东莞爱尔眼科医院","深圳华厦眼科医院","大连何氏眼科医院","江门爱尔新希塑眼科医","爱尔康爱眼俱乐部","重庆华厦眼科医院","杭州柠盟眼科","天津普瑞眼科医院","宁波爱尔眼科医院","成都华厦眼科医院","天津爱尔眼科医院","西安爱尔眼科医院","珠海希玛林顺潮眼科医区","广州番禺普瑞眼科医院","佛山爱尔眼科医院","红谷滩普瑞眼科光中心","深圳黑马眼科医院","深圳佰视住眼科","郑州视光眼科医院","南昌爱尔眼科医院","重庆爱尔眼科南坪医院","沃瑞眼科","淮安光明眼科医院","未来视界少儿眼科","武汉爱尔眼科医院","武汉爱尔眼科医院汉口","画美高级美学","德医专业皮肤医美","广州华美医疗美容医院","奇估医美","伯思立修复美学","Yestar深圳艺星","阳光医美整形颜究院","广州紫馨","广州曙光抗衰逆龄中心","Yestar艺星医美","伯思立医疗美学","华美紫馨颜究所","鹏爱医美国际","北京小壹美","Yestar艺星医美中心","肤研坊FAVOURFOUN","美莱医美全国连锁","深圳米兰柏羽","北京薇琳医美","华美紫馨医学美容中心","氧颜蔻抗衰中心","GIVRE绮芙莉","Yestar杭州艺星","深圳联合丽格","北京华臣眼博士医疗美容","ClearK洁铭品质医美","Yestar艺星医疗美容","北京英煌","Yeur艺星颜选","艺星皮肤抗衰中心","北京联合丽格第一医院","西安画美医疗美容医","上海薇琳轮廓美学","天津美莱","漾格医美","广州紫馨","今识理想医美","阳光医美轮常抗套中心","上海伊菜美形","广州氧颜优选","河南整形美容医院轮美学","原肌美塑品质医美","晶肤医疗美容医院","美诗沁专注去眼袋","南医大友谊美学抗衰","原肌美塑","北京欣扬医美","贵阳华美紫馨整形美","Yer艺星医疗美容医院","亚太医美面部抗衰","云南华美美菜医学美容中心","米杨丽格","艺星抗衰美肤中心","Yestar艺星美学","德医皮肤","西安艺星","保定蓝山医美","画美医疗抗衰中心","贵阳美贝尔医美","Yestar艺星医学美容","Redrcam宁波薇琳","北京画美艺术美学","北京和睦家医院","Yestar成都艺星","Yestar艺星皮肤美容中","宁波美莱","艺星抗衰中心","苏州美贝尔医美","BR光电保养上海","Yestar艺星美容基地","河南整形美容医院","Yestar黑龙江艺星","艺星医学美容中心","环球美天医疗美容","艺星皮肤美容","深圳小珂丽格","烟台鹏爱","黑龙江艺星","石家庄美莱","艺星皮肤美容中心","海口壹加壹美容","JUVA欧华轻医美连锁","JUVA欧华医美","REDREAM薇琳美学","大连美天医美","宁波薇琳抗衰美肤中","四川美莱医学美容","爱德丽格","南医大友谊无锡抗衰","四川悦好皮肤美容中","艺星美容","杭州美莱","Yestar艺星气质微整","衡阳美莱医美","威兹曼湖州曙光","Yestar艺星医疗美容中","美天变美课堂","Mylike医美严选","杭州薇琳植发","薇琳美容祛班中心","福州爱康国宾","爱帝宫月子中心","圣贝拉母要护理中心","爱帝官月子会所","babybella小贝拉月子中4","宝生月子中心","北京爱帝宫月子中心","深圳爱帝官月子中心","成都爱帝官月子会所","深圳爱帝宫月子会所","成都爱帝宫月子中心","北京爱帝官月子会所","宝生月子会所","和伊母婴护理中心","贵州天恩母要月子中心","月阁月子中心","上海爱尚家月子中心","积木月子中心","玥紫官元月子中心","碧莲盛植发·莲主任","碧莲盛植发·脱发养护发","碧莲盛不剃发植发医院","雍禾植发医院官方","碧莲盛不剃发植发","大麦微针美学植发","雍禾植发脱发医院","碧莲盛植发·养发全国连锁","雍禾植发小助理","碧莲盛毛发种植中心","碧莲盛·发际线美学","碧莲盛植发咨询中心","碧莲盛美学植发","雍禾发之初女性美学植发","碧莲盛不剃发植发","碧莲盛植发活动中心","老袁·碧莲盛植发","碧莲盛植发线上所动中心","贞韩植发","碧莲盛植发福利中心","雍禾植发","大麦微针植发","老蒋论植发.碧莲盛","碧莲盛不剃发·舒适","北京美中宜和妇儿医院","美中宜和北三环妇儿医院","福州伽禾伽美医院","山西尔彩医院","徐鲁·雍禾植发","高新米兰柏羽","雍禾毛发中心","雍禾不剃发植发医院","广州曙光医学美容医院","雍禾植发医院","伊美尔","mylike漾格医美","四川华美紫馨医疗美容","雍禾毛发医院","碧莲盛植发全国连锁"};
        String[] deparr = new String[]{"口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","口腔科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","眼科","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","医美","体检中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","月子中心","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","植发","妇产科","妇产科","妇产科","妇产科","植发","医美","植发","植发","医美","植发","医美","医美","医美","植发","植发"};

        OkHttpClient client = new OkHttpClient().newBuilder()
                // 设置连接超时为 60 秒
                .connectTimeout(60, TimeUnit.SECONDS)
                // 设置读取超时为 60 秒
                .readTimeout(60, TimeUnit.SECONDS)
                // 设置写入超时为 60 秒
                .writeTimeout(10, TimeUnit.SECONDS)
                .build();
        MediaType mediaType = MediaType.parse("application/json;charset=UTF-8");
        List<Map<String,String>> feedList = new ArrayList<>();

        List<String[]> titleAliasList = new ArrayList<>();
        titleAliasList.add(new String[]{"keyword", "关键词"});
        titleAliasList.add(new String[]{"department", "科室"});
        titleAliasList.add(new String[]{"url", "FeedUrl"});
        titleAliasList.add(new String[]{"title", "文章标题"});
        titleAliasList.add(new String[]{"desc", "文案描述"});
        titleAliasList.add(new String[]{"likedCount", "点赞数"});
        titleAliasList.add(new String[]{"collectedCount", "收藏数"});
        titleAliasList.add(new String[]{"commentCount", "评论数"});
        titleAliasList.add(new String[]{"shareCount", "分享数"});

        for (int n = 0; n < arr.length; n++) {
            feedList.clear();
            if(n!=0) {
                Thread.sleep(1000 * 60 * 5);
            }
            int i = 0;
            boolean hasMore = true;
            while (hasMore) {
                i++;
                RequestBody body = RequestBody.create(mediaType, "{\"keyword\":\"" + arr[n] + "\",\"page\":" + i + ",\"page_size\":20,\"search_id\":\"" + SEARCH_ID + "\",\"sort\":\"general\",\"note_type\":0,\"ext_flags\":[],\"image_formats\":[\"jpg\",\"webp\",\"avif\"]}");
                String paramJsonStr = "{\"url\": \"/api/sns/web/v1/search/notes\",\"b1\": \"" + B1 + "\",\"data\":\"{\\\"keyword\\\": \\\"" + arr[n] + "\\\",\\\"page\\\": " + i + ",\\\"page_size\\\": 20,\\\"search_id\\\": \\\"" + SEARCH_ID + "\\\",\\\"sort\\\": \\\"general\\\",\\\"note_type\\\": 0,\\\"ext_flags\\\": [],\\\"image_formats\\\": [\\\"jpg\\\",\\\"webp\\\",\\\"avif\\\"]}\"}";
                Map<String, String> xsMap = XiaohongshuCrawler.getXs(paramJsonStr,"keyword");
                if (xsMap.isEmpty() && xsMap.get("xs") != null) {
                    throw new Exception("获取xsMap失败");
                }
                Request request = new Request.Builder()
                        .url("https://edith.xiaohongshu.com/api/sns/web/v1/search/notes")
                        .method("POST", body)
                        .addHeader("accept", "application/json, text/plain, */*")
                        .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
                        .addHeader("content-type", "application/json;charset=UTF-8")
                        .addHeader("cookie", "abRequestId=4688e422-3bd2-589f-b891-fe4fcd33a55f; a1=1936108ad9bcw9vid6fgm3gndytwknc7j4eql9bg050000261389; webId=29dad20020f97cab8cafc205030d714b; gid=yjqKy8YDWjViyjqKy8Y0f4jDjDSEjh3fKik6dk8qkIfv9A28ETISW1888JKyqYj88JqK2DjD; web_session=040069b1ad2a33d0354c5eed73354be54667cc; customer-sso-sid=68c51744260276408322232751dae96f2f9f58e0; x-user-id-partner.xiaohongshu.com=67496db54f00000000000002; customerClientId=826692150613376; adslogin-access-token-xue.xiaohongshu.com=customer.xue.AT-390f6cbd28c841f0bdfaa34e6c6ebc8ea8d190fdfeea4c8ab7b6ca3e212e8aca; webBuild=4.45.1; xsecappid=xhs-pc-web; unread={%22ub%22:%22674a5c8c0000000002019ad1%22%2C%22ue%22:%226749df1500000000020191b2%22%2C%22uc%22:25}; acw_tc=0a0b139217331091246152375e28db376eb2e27acc2d44b4a131851b6945ec; websectiga=cffd9dcea65962b05ab048ac76962acee933d26157113bb213105a116241fa6c; sec_poison_id=2c34c1ba-a6da-48b9-a6c4-2b4cd37fa071")
                        .addHeader("origin", "https://www.xiaohongshu.com")
                        .addHeader("priority", "u=1, i")
                        .addHeader("referer", "https://www.xiaohongshu.com/")
                        .addHeader("sec-ch-ua", "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"")
                        .addHeader("sec-ch-ua-mobile", "?0")
                        .addHeader("sec-ch-ua-platform", "\"Windows\"")
                        .addHeader("sec-fetch-dest", "empty")
                        .addHeader("sec-fetch-mode", "cors")
                        .addHeader("sec-fetch-site", "same-site")
                        .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
                        .addHeader("x-b3-traceid", "f15e7b1d9ad548e8")
                        .addHeader("x-xray-traceid", "c9c2b1ab2a32ac59ff07e43a03ff2140")
                        .addHeader("x-s", xsMap.get("xs").toString())
                        .addHeader("x-s-common", xsMap.get("xsCommon").toString())
                        .addHeader("x-t", xsMap.get("xt").toString())
                        .build();

                Response response = client.newCall(request).execute();
                String rsponseStr = response.body().string();
                System.out.println("notes===:" + rsponseStr);
                JSONObject resObj = JSON.parseObject(rsponseStr);
                String code = resObj.getString("code");
                if (!"0".equals(code)) {
                    continue;
                }
                JSONObject dataJson = resObj.getJSONObject("data");
                JSONArray itemsJson = dataJson.getJSONArray("items");
                hasMore = dataJson.getBoolean("has_more");
                System.out.println("hasMore====" + hasMore + "----" + i);
                if (!hasMore) {
                    break;
                }
                for (int j = 0; j < itemsJson.size(); j++) {
                    if (j == 1) {
                        continue;
                    }
                    JSONObject itemObj = JSON.parseObject(itemsJson.get(j).toString());
                    String id = itemObj.get("id").toString();
                    String xsecToken = itemObj.get("xsec_token").toString();
//                String title = itemObj.getJSONObject("note_card").getString("display_title");
//                title = title==null?"":title;
                    HashMap<String, String> feedMap = getFeedInfo(id, xsecToken,arr[n],deparr[n],"","keyword");
                    if (feedMap != null) {
                        feedList.add(feedMap);
                    }

                }

            }

            ExcelUtils2.exportLocal(titleAliasList, feedList, "xiaohongshu", "小红书数据", arr[n] + ".xlsx", null);
            generalSql(feedList,arr[n]+".sql","0");
        }
    }
}
